//
// SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
//
// SPDX-License-Identifier: Apache-2.0
//

// Do not flag up inline assembly blocks
#pragma GCC diagnostic ignored "-Woverlength-strings"

#if !defined(__aarch64__) || !defined(__ARM_FEATURE_SVE2)
#error This file must be compiled for AArch64, FEAT_SVE2.
#else  // Architectural features check.

#include "kai_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot.h"

#include <stddef.h>
#include <stdint.h>

#include "kai/kai_common.h"

static const size_t kai_m_step = 1;
static const size_t kai_nr = 2;
static const size_t kai_n_step = 16;
static const size_t kai_kr = 4;
static const size_t kai_sr = 1;

size_t kai_get_m_step_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(void) {
    return kai_m_step;
}

size_t kai_get_n_step_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(void) {
    return kai_n_step * kai_get_sme_vector_length_u8() / kai_kr;
}

size_t kai_get_nr_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(void) {
    return kai_nr * kai_get_sme_vector_length_u8() / kai_kr;
}

size_t kai_get_kr_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(void) {
    return kai_kr;
}

size_t kai_get_sr_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(void) {
    return kai_sr;
}

size_t kai_get_lhs_offset_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(size_t m_idx, size_t k) {
    KAI_ASSUME(m_idx == 0);

    return m_idx * k;
}

static size_t kai_get_rhs_packed_stride_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(size_t k) {
    return kai_get_n_step_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot() *
        (kai_roundup(k, kai_kr) * sizeof(int8_t) + sizeof(int32_t) + sizeof(int32_t));
}

size_t kai_get_rhs_packed_offset_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(size_t n_idx, size_t k) {
    KAI_ASSUME(n_idx % kai_get_n_step_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot() == 0);

    const size_t block_idx = n_idx / kai_get_n_step_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot();
    return block_idx * kai_get_rhs_packed_stride_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(k);
}

size_t kai_get_dst_offset_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(
    size_t m_idx, size_t n_idx, size_t dst_stride) {
    KAI_ASSUME(m_idx == 0);
    KAI_ASSUME(n_idx % kai_get_n_step_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot() == 0);

    return (m_idx * dst_stride) + (n_idx * sizeof(int8_t));
}

size_t kai_get_dst_size_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(size_t m, size_t n) {
    return m * n * sizeof(int8_t);
}

void kai_run_matmul_clamp_qai8_qai8_qsi8cxp2vlx4sb_1x16vl_sme2_dot(
    size_t m, size_t n, size_t k, const void* lhs, const void* rhs_packed, void* dst, size_t dst_stride_row,
    size_t dst_stride_col, const struct kai_matmul_requantize32_params* params) {
    KAI_UNUSED(dst_stride_row);
    KAI_UNUSED(dst_stride_col);
    KAI_ASSUME(m == 1);

    typedef struct {
        int32_t c_offset;
        int32_t maxval;
        int32_t minval;
    } KernelArgs;

    KernelArgs k_args;
    k_args.maxval = params->max_value;
    k_args.minval = params->min_value;
    k_args.c_offset = params->output_zero_point;

    size_t N = n;
    size_t K = k;

    const void* A_ptr = lhs;
    const void* B_ptr = rhs_packed;
    void* output_ptr = dst;

    uint64_t flags = 0;

    __asm__ __volatile__(
        ".inst 0xd503477f  // SMSTART ZA\n"
        "mov x8, #0x0\n"
        "mov x16, %x[B_ptr]\n"
        "cntw x15, ALL, MUL #4\n"
        "mov x14, %x[output_ptr]\n"
        "add x13, %x[N], x15\n"
        "ptrue p2.b\n"
        "sub x13, x13, #0x1\n"
        ".inst 0x25207810  // ptrue pn8.b\n"
        "udiv x13, x13, x15\n"
        "mov x22, #0x1\n"
        "add x21, x13, #0x3\n"
        "and x21, x21, #0xfffffffffffffffc\n"
        "mul x21, x21, x15\n"
        "mul x21, x21, %x[K]\n"
        "1:"  // RHS size check loop
        "cmp x21, #0x200000\n"
        "blt 2f\n"
        "tbnz x21, #0, 3f\n"
        "lsr x21, x21, #0x1\n"
        "lsl x22, x22, #0x1\n"
        "b 1b\n"
        "2:"  // RHS do prefetch
        "lsl x20, x21, #0x26\n"
        "sub x22, x22, #0x1\n"
        "lsl x22, x22, #0x16\n"
        "orr x21, x21, x20\n"
        "orr x21, x21, x22\n"
        ".inst 0xf8b54a1a  // rprfm pldonce, x21, [x16]\n"
        "3:"  // RHS prefetch exit
        "add x12, %x[K], #0x3\n"
        "cntw x20, ALL, MUL #2\n"
        "mov z25.s, #0x0\n"
        "mov z27.b, #0x1\n"
        "bic x12, x12, #0x3\n"
        "bic %x[flags], %x[flags], #0x80000000\n"
        "add x12, x12, #0x8\n"
        "mul x12, x12, x20\n"
        "4:"  // Column loop
        "cmp x13, #0x4\n"
        "bge 25f\n"
        "cmp x13, #0x2\n"
        "bgt 18f\n"
        "beq 11f\n"
        "cntw x20, ALL, MUL #2\n"
        "add x23, x16, x12\n"
        ".inst 0xa0404210  // ld1w { z16.s-z17.s }, pn8.b/Z, [x16]\n"
        "cmp %x[N], x20\n"
        "mov x11, %x[K]\n"
        "csel x23, x23, x16, GT\n"
        "mov x21, %x[N]\n"
        ".inst 0xa04042f2  // ld1w { z18.s-z19.s }, pn8.b/Z, [x23]\n"
        "mov x10, %x[A_ptr]\n"
        "mov x20, %x[K]\n"
        "whilelt p1.b, XZR, x21\n"
        "cmp x11, #0x10\n"
        ".inst 0xf8b44958  // rprfm pldmany, x20, [x10]\n"
        "addvl x16, x16, #2\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc0040e00  // mova za.d[x8, #0], { z16.d-z19.d }\n"
        "ble 7f\n"
        "5:"  // Width 1: Multiply loop: Main loop head
        "whilelt p0.b, XZR, x11\n"
        ".inst 0xa040021d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        "ld1rqb { z13.b }, p0/Z, [x10]\n"
        "add x10, x10, #0x10\n"
        ".inst 0xa04002ff  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa0400215  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002f7  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d93a0  // sdot za.s[x8, 0], { z28.b-z31.b }, z13.b[0]\n"
        ".inst 0xa0400211  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002f3  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa040021d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002ff  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d96a0  // sdot za.s[x8, 0], { z20.b-z23.b }, z13.b[1]\n"
        ".inst 0xc15d9a20  // sdot za.s[x8, 0], { z16.b-z19.b }, z13.b[2]\n"
        ".inst 0xc15d9fa0  // sdot za.s[x8, 0], { z28.b-z31.b }, z13.b[3]\n"
        "tbnz %x[flags], #31, 6f\n"
        "sdot z25.s, z13.b, z27.b\n"
        "6:"  // Width 1: Multiply loop: unique 1: skip row sum
        "sub x11, x11, #0x10\n"
        "cmp x11, #0x10\n"
        "bgt 5b\n"
        "7:"  // Width 1: Multiply loop: Single iteration only
        "whilelt p0.b, XZR, x11\n"
        ".inst 0xa0400205  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "ld1rqb { z13.b }, p0/Z, [x10]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002e7  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d90a0  // sdot za.s[x8, 0], { z4.b-z7.b }, z13.b[0]\n"
        "ble 8f\n"
        ".inst 0xa040021d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002ff  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d97a0  // sdot za.s[x8, 0], { z28.b-z31.b }, z13.b[1]\n"
        "ble 8f\n"
        ".inst 0xa0400209  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002eb  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d9920  // sdot za.s[x8, 0], { z8.b-z11.b }, z13.b[2]\n"
        "ble 8f\n"
        ".inst 0xa040021d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002ff  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d9fa0  // sdot za.s[x8, 0], { z28.b-z31.b }, z13.b[3]\n"
        "8:"  // Width 1: Multiply loop: multiply skip
        "tbnz %x[flags], #31, 9f\n"
        "9:"  // Width 1: Multiply loop: unique 2: skip row sum
        ".inst 0xc0060c08  // mova { z8.d-z11.d }, za.d[x8, #0]\n"
        ".inst 0xa040421e  // ld1w { z30.s-z31.s }, pn8.b/Z, [x16]\n"
        "add x22, %x[k_args], %[c_offset]\n"
        "add x21, %x[k_args], %[minval]\n"
        ".inst 0xa04042f8  // ld1w { z24.s-z25.s }, pn8.b/Z, [x23]\n"
        "add x20, %x[k_args], %[maxval]\n"
        "ld1rw { z2.s }, p2/Z, [x22]\n"
        "ld1rw { z13.s }, p2/Z, [x21]\n"
        ".inst 0xc132e108  // scvtf { z8.s-z11.s }, { z8.s-z11.s }\n"
        "ld1rw { z20.s }, p2/Z, [x20]\n"
        "fmul z8.s, z8.s, z30.s\n"
        "fmul z9.s, z9.s, z31.s\n"
        "fmul z10.s, z10.s, z24.s\n"
        "fmul z11.s, z11.s, z25.s\n"
        ".inst 0xc1b8e108  // frintn { z8.s-z11.s }, { z8.s-z11.s }\n"
        ".inst 0xc131e108  // fcvtzs { z8.s-z11.s }, { z8.s-z11.s }\n"
        ".inst 0xc1a2ab08  // add { z8.s-z11.s }, { z8.s-z11.s }, z2.s\n"
        ".inst 0xc1b4cda8  // sclamp { z8.s-z11.s }, z13.s, z20.s\n"
        "uzp1 z8.h, z8.h, z9.h\n"
        "uzp1 z0.h, z10.h, z11.h\n"
        "uzp1 z8.b, z8.b, z0.b\n"
        "st1b { z8.b }, p1, [x14]\n"
        "b 32f\n"
        "11:"  // Width 2
        "add x24, x16, x12, LSL #1\n"
        "cntw x20, ALL, MUL #6\n"
        ".inst 0xa0404214  // ld1w { z20.s-z21.s }, pn8.b/Z, [x16]\n"
        "add x22, x24, x12\n"
        "cmp %x[N], x20\n"
        ".inst 0xa040430c  // ld1w { z12.s-z13.s }, pn8.b/Z, [x24]\n"
        "add x23, x16, x12\n"
        "csel x22, x22, x16, GT\n"
        ".inst 0xa04042f6  // ld1w { z22.s-z23.s }, pn8.b/Z, [x23]\n"
        "mov x11, %x[K]\n"
        "sub x21, %x[N], x15\n"
        ".inst 0xa04042ce  // ld1w { z14.s-z15.s }, pn8.b/Z, [x22]\n"
        "mov x10, %x[A_ptr]\n"
        "mov x20, %x[K]\n"
        "whilelt p1.b, XZR, x21\n"
        "cmp x11, #0x10\n"
        ".inst 0xf8b44958  // rprfm pldmany, x20, [x10]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xc0040e80  // mova za.d[x8, #0], { z20.d-z23.d }\n"
        "addvl x23, x23, #2\n"
        "addvl x24, x24, #2\n"
        ".inst 0xc0040d81  // mova za.d[x8, #1], { z12.d-z15.d }\n"
        "addvl x22, x22, #2\n"
        "ble 14f\n"
        "12:"  // Width 2: Multiply loop: Main loop head
        "whilelt p0.b, XZR, x11\n"
        ".inst 0xa0400211  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        "ld1rqb { z13.b }, p0/Z, [x10]\n"
        "add x10, x10, #0x10\n"
        ".inst 0xa04002f3  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa0400305  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa04002c7  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xc15d9220  // sdot za.s[x8, 0], { z16.b-z19.b }, z13.b[0]\n"
        ".inst 0xa0400209  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002eb  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d90a1  // sdot za.s[x8, 1], { z4.b-z7.b }, z13.b[0]\n"
        ".inst 0xa0400311  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa04002d3  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xc15d9520  // sdot za.s[x8, 0], { z8.b-z11.b }, z13.b[1]\n"
        ".inst 0xa0400201  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002e3  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d9621  // sdot za.s[x8, 1], { z16.b-z19.b }, z13.b[1]\n"
        ".inst 0xa0400305  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa04002c7  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xc15d9820  // sdot za.s[x8, 0], { z0.b-z3.b }, z13.b[2]\n"
        ".inst 0xa040021d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002ff  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d98a1  // sdot za.s[x8, 1], { z4.b-z7.b }, z13.b[2]\n"
        ".inst 0xa0400309  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa04002cb  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xc15d9fa0  // sdot za.s[x8, 0], { z28.b-z31.b }, z13.b[3]\n"
        ".inst 0xc15d9d21  // sdot za.s[x8, 1], { z8.b-z11.b }, z13.b[3]\n"
        "tbnz %x[flags], #31, 13f\n"
        "sdot z25.s, z13.b, z27.b\n"
        "13:"  // Width 2: Multiply loop: unique 3: skip row sum
        "sub x11, x11, #0x10\n"
        "cmp x11, #0x10\n"
        "bgt 12b\n"
        "14:"  // Width 2: Multiply loop: Single iteration only
        "whilelt p0.b, XZR, x11\n"
        ".inst 0xa0400211  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "ld1rqb { z13.b }, p0/Z, [x10]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002f3  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa0400301  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa04002c3  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xc15d9220  // sdot za.s[x8, 0], { z16.b-z19.b }, z13.b[0]\n"
        ".inst 0xc15d9021  // sdot za.s[x8, 1], { z0.b-z3.b }, z13.b[0]\n"
        "ble 15f\n"
        ".inst 0xa0400215  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002f7  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa040031d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa04002df  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xc15d96a0  // sdot za.s[x8, 0], { z20.b-z23.b }, z13.b[1]\n"
        ".inst 0xc15d97a1  // sdot za.s[x8, 1], { z28.b-z31.b }, z13.b[1]\n"
        "ble 15f\n"
        ".inst 0xa0400201  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002e3  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa0400319  // ldnt1b { z24.b-z25.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa04002db  // ldnt1b { z26.b-z27.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xc15d9820  // sdot za.s[x8, 0], { z0.b-z3.b }, z13.b[2]\n"
        ".inst 0xc15d9b21  // sdot za.s[x8, 1], { z24.b-z27.b }, z13.b[2]\n"
        "ble 15f\n"
        ".inst 0xa0400219  // ldnt1b { z24.b-z25.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002fb  // ldnt1b { z26.b-z27.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa040031d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x24]\n"
        ".inst 0xa04002df  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x22]\n"
        ".inst 0xc15d9f20  // sdot za.s[x8, 0], { z24.b-z27.b }, z13.b[3]\n"
        ".inst 0xc15d9fa1  // sdot za.s[x8, 1], { z28.b-z31.b }, z13.b[3]\n"
        "15:"  // Width 2: Multiply loop: multiply skip
        "tbnz %x[flags], #31, 16f\n"
        "16:"  // Width 2: Multiply loop: unique 4: skip row sum
        ".inst 0xc0060c00  // mova { z0.d-z3.d }, za.d[x8, #0]\n"
        ".inst 0xa0404208  // ld1w { z8.s-z9.s }, pn8.b/Z, [x16]\n"
        "add x22, %x[k_args], %[c_offset]\n"
        "add x21, %x[k_args], %[minval]\n"
        ".inst 0xa04042fe  // ld1w { z30.s-z31.s }, pn8.b/Z, [x23]\n"
        "add x20, %x[k_args], %[maxval]\n"
        ".inst 0xc0060c24  // mova { z4.d-z7.d }, za.d[x8, #1]\n"
        "add x16, x16, x12, LSL #1\n"
        "ld1rw { z14.s }, p2/Z, [x22]\n"
        "add x23, x23, x12, LSL #1\n"
        "ld1rw { z11.s }, p2/Z, [x21]\n"
        ".inst 0xc132e000  // scvtf { z0.s-z3.s }, { z0.s-z3.s }\n"
        "ld1rw { z10.s }, p2/Z, [x20]\n"
        "fmul z0.s, z0.s, z8.s\n"
        "fmul z1.s, z1.s, z9.s\n"
        ".inst 0xc132e084  // scvtf { z4.s-z7.s }, { z4.s-z7.s }\n"
        "fmul z2.s, z2.s, z30.s\n"
        "fmul z3.s, z3.s, z31.s\n"
        ".inst 0xc1b8e000  // frintn { z0.s-z3.s }, { z0.s-z3.s }\n"
        ".inst 0xc131e000  // fcvtzs { z0.s-z3.s }, { z0.s-z3.s }\n"
        ".inst 0xc1aeab00  // add { z0.s-z3.s }, { z0.s-z3.s }, z14.s\n"
        ".inst 0xc1aacd60  // sclamp { z0.s-z3.s }, z11.s, z10.s\n"
        "uzp1 z0.h, z0.h, z1.h\n"
        "uzp1 z16.h, z2.h, z3.h\n"
        "uzp1 z0.b, z0.b, z16.b\n"
        "st1b { z0.b }, p2, [x14]\n"
        ".inst 0xa1404217  // ld1w { z23.s, z31.s }, pn8.b/Z, [x16]\n"
        ".inst 0xa14042f6  // ld1w { z22.s, z30.s }, pn8.b/Z, [x23]\n"
        "fmul z4.s, z4.s, z23.s\n"
        "fmul z5.s, z5.s, z31.s\n"
        "fmul z6.s, z6.s, z22.s\n"
        "fmul z7.s, z7.s, z30.s\n"
        ".inst 0xc1b8e084  // frintn { z4.s-z7.s }, { z4.s-z7.s }\n"
        ".inst 0xc131e084  // fcvtzs { z4.s-z7.s }, { z4.s-z7.s }\n"
        ".inst 0xc1aeab04  // add { z4.s-z7.s }, { z4.s-z7.s }, z14.s\n"
        ".inst 0xc1aacd64  // sclamp { z4.s-z7.s }, z11.s, z10.s\n"
        "uzp1 z4.h, z4.h, z5.h\n"
        "uzp1 z2.h, z6.h, z7.h\n"
        "uzp1 z4.b, z4.b, z2.b\n"
        "st1b { z4.b }, p1, [x14, #1, MUL VL]\n"
        "b 32f\n"
        "18:"  // Width 3
        "add x26, x16, x12, LSL #2\n"
        "cntw x20, ALL, MUL #10\n"
        ".inst 0xa0404210  // ld1w { z16.s-z17.s }, pn8.b/Z, [x16]\n"
        "add x25, x16, x12, LSL #1\n"
        "add x24, x26, x12\n"
        ".inst 0xa040435c  // ld1w { z28.s-z29.s }, pn8.b/Z, [x26]\n"
        "cmp %x[N], x20\n"
        "add x23, x16, x12\n"
        ".inst 0xa040432c  // ld1w { z12.s-z13.s }, pn8.b/Z, [x25]\n"
        "add x22, x25, x12\n"
        "csel x24, x24, x16, GT\n"
        ".inst 0xa04042f2  // ld1w { z18.s-z19.s }, pn8.b/Z, [x23]\n"
        "mov x20, #0x2\n"
        ".inst 0xa04042ce  // ld1w { z14.s-z15.s }, pn8.b/Z, [x22]\n"
        "mov x11, %x[K]\n"
        ".inst 0xa040431e  // ld1w { z30.s-z31.s }, pn8.b/Z, [x24]\n"
        "msub x21, x15, x20, %x[N]\n"
        "mov x10, %x[A_ptr]\n"
        "mov x20, %x[K]\n"
        "whilelt p1.b, XZR, x21\n"
        ".inst 0xc0040e00  // mova za.d[x8, #0], { z16.d-z19.d }\n"
        "cmp x11, #0x10\n"
        ".inst 0xf8b44958  // rprfm pldmany, x20, [x10]\n"
        ".inst 0xc0040d81  // mova za.d[x8, #1], { z12.d-z15.d }\n"
        "addvl x16, x16, #2\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc0040f82  // mova za.d[x8, #2], { z28.d-z31.d }\n"
        "addvl x25, x25, #2\n"
        "addvl x22, x22, #2\n"
        "addvl x26, x26, #2\n"
        "addvl x24, x24, #2\n"
        "ble 21f\n"
        "19:"  // Width 3: Multiply loop: Main loop head
        "whilelt p0.b, XZR, x11\n"
        ".inst 0xa0400201  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        "ld1rqb { z13.b }, p0/Z, [x10]\n"
        "add x10, x10, #0x10\n"
        ".inst 0xa04002e3  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa0400329  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa04002cb  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400351  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d9020  // sdot za.s[x8, 0], { z0.b-z3.b }, z13.b[0]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xa0400313  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xc15d9121  // sdot za.s[x8, 1], { z8.b-z11.b }, z13.b[0]\n"
        ".inst 0xa040021d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002ff  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d9222  // sdot za.s[x8, 2], { z16.b-z19.b }, z13.b[0]\n"
        ".inst 0xa0400321  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa04002c3  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400345  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d97a0  // sdot za.s[x8, 0], { z28.b-z31.b }, z13.b[1]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xa0400307  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xc15d9421  // sdot za.s[x8, 1], { z0.b-z3.b }, z13.b[1]\n"
        ".inst 0xa0400215  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002f7  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d94a2  // sdot za.s[x8, 2], { z4.b-z7.b }, z13.b[1]\n"
        ".inst 0xa040033d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa04002df  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400345  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d9aa0  // sdot za.s[x8, 0], { z20.b-z23.b }, z13.b[2]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xa0400307  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xc15d9ba1  // sdot za.s[x8, 1], { z28.b-z31.b }, z13.b[2]\n"
        ".inst 0xa040021d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002ff  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xc15d98a2  // sdot za.s[x8, 2], { z4.b-z7.b }, z13.b[2]\n"
        ".inst 0xa0400335  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa04002d7  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400351  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d9fa0  // sdot za.s[x8, 0], { z28.b-z31.b }, z13.b[3]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xa0400313  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xc15d9ea1  // sdot za.s[x8, 1], { z20.b-z23.b }, z13.b[3]\n"
        ".inst 0xc15d9e22  // sdot za.s[x8, 2], { z16.b-z19.b }, z13.b[3]\n"
        "tbnz %x[flags], #31, 20f\n"
        "sdot z25.s, z13.b, z27.b\n"
        "20:"  // Width 3: Multiply loop: unique 5: skip row sum
        "sub x11, x11, #0x10\n"
        "cmp x11, #0x10\n"
        "bgt 19b\n"
        "21:"  // Width 3: Multiply loop: Single iteration only
        "whilelt p0.b, XZR, x11\n"
        ".inst 0xa0400211  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "ld1rqb { z13.b }, p0/Z, [x10]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002f3  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa0400325  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa04002c7  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa040035d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d9220  // sdot za.s[x8, 0], { z16.b-z19.b }, z13.b[0]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xa040031f  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xc15d90a1  // sdot za.s[x8, 1], { z4.b-z7.b }, z13.b[0]\n"
        ".inst 0xc15d93a2  // sdot za.s[x8, 2], { z28.b-z31.b }, z13.b[0]\n"
        "ble 22f\n"
        ".inst 0xa0400211  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002f3  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa0400329  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa04002cb  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400345  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d9620  // sdot za.s[x8, 0], { z16.b-z19.b }, z13.b[1]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xa0400307  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xc15d9521  // sdot za.s[x8, 1], { z8.b-z11.b }, z13.b[1]\n"
        ".inst 0xc15d94a2  // sdot za.s[x8, 2], { z4.b-z7.b }, z13.b[1]\n"
        "ble 22f\n"
        ".inst 0xa0400209  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002eb  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa040033d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa04002df  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400359  // ldnt1b { z24.b-z25.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d9920  // sdot za.s[x8, 0], { z8.b-z11.b }, z13.b[2]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xa040031b  // ldnt1b { z26.b-z27.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xc15d9ba1  // sdot za.s[x8, 1], { z28.b-z31.b }, z13.b[2]\n"
        ".inst 0xc15d9b22  // sdot za.s[x8, 2], { z24.b-z27.b }, z13.b[2]\n"
        "ble 22f\n"
        ".inst 0xa040021d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa04002ff  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x23]\n"
        "addvl x23, x23, #2\n"
        ".inst 0xa0400339  // ldnt1b { z24.b-z25.b }, pn8.b/Z, [x25]\n"
        ".inst 0xa04002db  // ldnt1b { z26.b-z27.b }, pn8.b/Z, [x22]\n"
        ".inst 0xa0400355  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d9fa0  // sdot za.s[x8, 0], { z28.b-z31.b }, z13.b[3]\n"
        ".inst 0xa0400317  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x24]\n"
        ".inst 0xc15d9f21  // sdot za.s[x8, 1], { z24.b-z27.b }, z13.b[3]\n"
        ".inst 0xc15d9ea2  // sdot za.s[x8, 2], { z20.b-z23.b }, z13.b[3]\n"
        "22:"  // Width 3: Multiply loop: multiply skip
        "tbnz %x[flags], #31, 23f\n"
        "23:"  // Width 3: Multiply loop: unique 6: skip row sum
        ".inst 0xc0060c18  // mova { z24.d-z27.d }, za.d[x8, #0]\n"
        ".inst 0xa0404202  // ld1w { z2.s-z3.s }, pn8.b/Z, [x16]\n"
        "add x22, %x[k_args], %[c_offset]\n"
        "add x21, %x[k_args], %[minval]\n"
        ".inst 0xa04042e6  // ld1w { z6.s-z7.s }, pn8.b/Z, [x23]\n"
        "add x20, %x[k_args], %[maxval]\n"
        ".inst 0xc0060c28  // mova { z8.d-z11.d }, za.d[x8, #1]\n"
        "add x16, x16, x12, LSL #1\n"
        "ld1rw { z0.s }, p2/Z, [x22]\n"
        "add x23, x23, x12, LSL #1\n"
        ".inst 0xc0060c5c  // mova { z28.d-z31.d }, za.d[x8, #2]\n"
        "ld1rw { z19.s }, p2/Z, [x21]\n"
        ".inst 0xc132e318  // scvtf { z24.s-z27.s }, { z24.s-z27.s }\n"
        "ld1rw { z18.s }, p2/Z, [x20]\n"
        "fmul z24.s, z24.s, z2.s\n"
        "fmul z25.s, z25.s, z3.s\n"
        ".inst 0xc132e108  // scvtf { z8.s-z11.s }, { z8.s-z11.s }\n"
        "fmul z26.s, z26.s, z6.s\n"
        "fmul z27.s, z27.s, z7.s\n"
        ".inst 0xc132e39c  // scvtf { z28.s-z31.s }, { z28.s-z31.s }\n"
        ".inst 0xc1b8e318  // frintn { z24.s-z27.s }, { z24.s-z27.s }\n"
        ".inst 0xc131e318  // fcvtzs { z24.s-z27.s }, { z24.s-z27.s }\n"
        ".inst 0xc1a0ab18  // add { z24.s-z27.s }, { z24.s-z27.s }, z0.s\n"
        ".inst 0xc1b2ce78  // sclamp { z24.s-z27.s }, z19.s, z18.s\n"
        "uzp1 z24.h, z24.h, z25.h\n"
        "uzp1 z16.h, z26.h, z27.h\n"
        "uzp1 z24.b, z24.b, z16.b\n"
        "st1b { z24.b }, p2, [x14]\n"
        ".inst 0xa1404207  // ld1w { z7.s, z15.s }, pn8.b/Z, [x16]\n"
        "add x16, x16, x12, LSL #1\n"
        ".inst 0xa14042f1  // ld1w { z17.s, z25.s }, pn8.b/Z, [x23]\n"
        "add x23, x23, x12, LSL #1\n"
        "fmul z8.s, z8.s, z7.s\n"
        "fmul z9.s, z9.s, z15.s\n"
        "fmul z10.s, z10.s, z17.s\n"
        "fmul z11.s, z11.s, z25.s\n"
        ".inst 0xc1b8e108  // frintn { z8.s-z11.s }, { z8.s-z11.s }\n"
        ".inst 0xc131e108  // fcvtzs { z8.s-z11.s }, { z8.s-z11.s }\n"
        ".inst 0xc1a0ab08  // add { z8.s-z11.s }, { z8.s-z11.s }, z0.s\n"
        ".inst 0xc1b2ce68  // sclamp { z8.s-z11.s }, z19.s, z18.s\n"
        "uzp1 z8.h, z8.h, z9.h\n"
        "uzp1 z16.h, z10.h, z11.h\n"
        "uzp1 z8.b, z8.b, z16.b\n"
        "st1b { z8.b }, p2, [x14, #1, MUL VL]\n"
        ".inst 0xa1404207  // ld1w { z7.s, z15.s }, pn8.b/Z, [x16]\n"
        ".inst 0xa14042f1  // ld1w { z17.s, z25.s }, pn8.b/Z, [x23]\n"
        "fmul z28.s, z28.s, z7.s\n"
        "fmul z29.s, z29.s, z15.s\n"
        "fmul z30.s, z30.s, z17.s\n"
        "fmul z31.s, z31.s, z25.s\n"
        ".inst 0xc1b8e39c  // frintn { z28.s-z31.s }, { z28.s-z31.s }\n"
        ".inst 0xc131e39c  // fcvtzs { z28.s-z31.s }, { z28.s-z31.s }\n"
        ".inst 0xc1a0ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n"
        ".inst 0xc1b2ce7c  // sclamp { z28.s-z31.s }, z19.s, z18.s\n"
        "uzp1 z28.h, z28.h, z29.h\n"
        "uzp1 z16.h, z30.h, z31.h\n"
        "uzp1 z28.b, z28.b, z16.b\n"
        "st1b { z28.b }, p1, [x14, #2, MUL VL]\n"
        "b 32f\n"
        "25:"  // Width 4
        "add x9, x16, x12, LSL #2\n"
        "cntw x20, ALL, MUL #14\n"
        ".inst 0xa040420c  // ld1w { z12.s-z13.s }, pn8.b/Z, [x16]\n"
        "add x28, x9, x12, LSL #1\n"
        "add x27, x16, x12, LSL #1\n"
        ".inst 0xa0404124  // ld1w { z4.s-z5.s }, pn8.b/Z, [x9]\n"
        "add x26, x28, x12\n"
        "cmp %x[N], x20\n"
        ".inst 0xa0404368  // ld1w { z8.s-z9.s }, pn8.b/Z, [x27]\n"
        "add x25, x16, x12\n"
        "add x24, x27, x12\n"
        ".inst 0xa0404380  // ld1w { z0.s-z1.s }, pn8.b/Z, [x28]\n"
        "add x22, x9, x12\n"
        "csel x26, x26, x16, GT\n"
        ".inst 0xa040432e  // ld1w { z14.s-z15.s }, pn8.b/Z, [x25]\n"
        "mov x20, #0x3\n"
        ".inst 0xa040430a  // ld1w { z10.s-z11.s }, pn8.b/Z, [x24]\n"
        "mov x11, %x[K]\n"
        ".inst 0xa04042c6  // ld1w { z6.s-z7.s }, pn8.b/Z, [x22]\n"
        "msub x21, x15, x20, %x[N]\n"
        "mov x10, %x[A_ptr]\n"
        ".inst 0xa0404342  // ld1w { z2.s-z3.s }, pn8.b/Z, [x26]\n"
        "mov x20, %x[K]\n"
        "whilelt p1.b, XZR, x21\n"
        ".inst 0xc0040d80  // mova za.d[x8, #0], { z12.d-z15.d }\n"
        "cmp x11, #0x10\n"
        ".inst 0xf8b44958  // rprfm pldmany, x20, [x10]\n"
        ".inst 0xc0040d01  // mova za.d[x8, #1], { z8.d-z11.d }\n"
        "add x23, x16, x12, LSL #3\n"
        "addvl x16, x16, #2\n"
        ".inst 0xc0040c82  // mova za.d[x8, #2], { z4.d-z7.d }\n"
        "addvl x25, x25, #2\n"
        "addvl x27, x27, #2\n"
        ".inst 0xc0040c03  // mova za.d[x8, #3], { z0.d-z3.d }\n"
        "addvl x24, x24, #2\n"
        "addvl x9, x9, #2\n"
        "addvl x22, x22, #2\n"
        "addvl x28, x28, #2\n"
        "addvl x26, x26, #2\n"
        "ble 28f\n"
        "26:"  // Width 4: Multiply loop: Main loop head
        "whilelt p0.b, XZR, x11\n"
        ".inst 0xa0400201  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        "ld1rqb { z13.b }, p0/Z, [x10]\n"
        "add x10, x10, #0x10\n"
        ".inst 0xa0400323  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa0400369  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x27]\n"
        "addvl x27, x27, #2\n"
        ".inst 0xa040030b  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa0400125  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x9]\n"
        ".inst 0xc15d9020  // sdot za.s[x8, 0], { z0.b-z3.b }, z13.b[0]\n"
        "addvl x9, x9, #2\n"
        ".inst 0xa04002c7  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400395  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x28]\n"
        ".inst 0xc15d9121  // sdot za.s[x8, 1], { z8.b-z11.b }, z13.b[0]\n"
        "addvl x28, x28, #2\n"
        ".inst 0xa0400357  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x26]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xc15d90a2  // sdot za.s[x8, 2], { z4.b-z7.b }, z13.b[0]\n"
        ".inst 0xa0400201  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa0400323  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xc15d92a3  // sdot za.s[x8, 3], { z20.b-z23.b }, z13.b[0]\n"
        ".inst 0xa0400365  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x27]\n"
        "addvl x27, x27, #2\n"
        ".inst 0xa0400307  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa0400131  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x9]\n"
        ".inst 0xc15d9420  // sdot za.s[x8, 0], { z0.b-z3.b }, z13.b[1]\n"
        "addvl x9, x9, #2\n"
        ".inst 0xa04002d3  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400381  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x28]\n"
        ".inst 0xc15d94a1  // sdot za.s[x8, 1], { z4.b-z7.b }, z13.b[1]\n"
        "addvl x28, x28, #2\n"
        ".inst 0xa0400343  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x26]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xc15d9622  // sdot za.s[x8, 2], { z16.b-z19.b }, z13.b[1]\n"
        ".inst 0xa0400215  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa0400337  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xc15d9423  // sdot za.s[x8, 3], { z0.b-z3.b }, z13.b[1]\n"
        ".inst 0xa0400371  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x27]\n"
        "addvl x27, x27, #2\n"
        ".inst 0xa0400313  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa0400125  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x9]\n"
        ".inst 0xc15d9aa0  // sdot za.s[x8, 0], { z20.b-z23.b }, z13.b[2]\n"
        "addvl x9, x9, #2\n"
        ".inst 0xa04002c7  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400381  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x28]\n"
        ".inst 0xc15d9a21  // sdot za.s[x8, 1], { z16.b-z19.b }, z13.b[2]\n"
        "addvl x28, x28, #2\n"
        ".inst 0xa0400343  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x26]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xc15d98a2  // sdot za.s[x8, 2], { z4.b-z7.b }, z13.b[2]\n"
        ".inst 0xa0400205  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa0400327  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xc15d9823  // sdot za.s[x8, 3], { z0.b-z3.b }, z13.b[2]\n"
        ".inst 0xa040037d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x27]\n"
        "addvl x27, x27, #2\n"
        ".inst 0xa040031f  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa0400135  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x9]\n"
        ".inst 0xc15d9ca0  // sdot za.s[x8, 0], { z4.b-z7.b }, z13.b[3]\n"
        "addvl x9, x9, #2\n"
        ".inst 0xa04002d7  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400385  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x28]\n"
        ".inst 0xc15d9fa1  // sdot za.s[x8, 1], { z28.b-z31.b }, z13.b[3]\n"
        "addvl x28, x28, #2\n"
        ".inst 0xa0400347  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x26]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xc15d9ea2  // sdot za.s[x8, 2], { z20.b-z23.b }, z13.b[3]\n"
        ".inst 0xc15d9ca3  // sdot za.s[x8, 3], { z4.b-z7.b }, z13.b[3]\n"
        "tbnz %x[flags], #31, 27f\n"
        "sdot z25.s, z13.b, z27.b\n"
        "27:"  // Width 4: Multiply loop: unique 7: skip row sum
        "sub x11, x11, #0x10\n"
        "cmp x11, #0x10\n"
        "bgt 26b\n"
        "28:"  // Width 4: Multiply loop: Single iteration only
        "whilelt p0.b, XZR, x11\n"
        ".inst 0xa0400205  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "ld1rqb { z13.b }, p0/Z, [x10]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa0400327  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa0400371  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x27]\n"
        "addvl x27, x27, #2\n"
        ".inst 0xa0400313  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa040013d  // ldnt1b { z28.b-z29.b }, pn8.b/Z, [x9]\n"
        ".inst 0xc15d90a0  // sdot za.s[x8, 0], { z4.b-z7.b }, z13.b[0]\n"
        "addvl x9, x9, #2\n"
        ".inst 0xa04002df  // ldnt1b { z30.b-z31.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400389  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x28]\n"
        ".inst 0xc15d9221  // sdot za.s[x8, 1], { z16.b-z19.b }, z13.b[0]\n"
        "addvl x28, x28, #2\n"
        ".inst 0xa040034b  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x26]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xc15d93a2  // sdot za.s[x8, 2], { z28.b-z31.b }, z13.b[0]\n"
        ".inst 0xc15d9123  // sdot za.s[x8, 3], { z8.b-z11.b }, z13.b[0]\n"
        "ble 29f\n"
        ".inst 0xa0400205  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa0400327  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa0400371  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x27]\n"
        "addvl x27, x27, #2\n"
        ".inst 0xa0400313  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa0400121  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x9]\n"
        ".inst 0xc15d94a0  // sdot za.s[x8, 0], { z4.b-z7.b }, z13.b[1]\n"
        "addvl x9, x9, #2\n"
        ".inst 0xa04002c3  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400385  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x28]\n"
        ".inst 0xc15d9621  // sdot za.s[x8, 1], { z16.b-z19.b }, z13.b[1]\n"
        "addvl x28, x28, #2\n"
        ".inst 0xa0400347  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x26]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xc15d9422  // sdot za.s[x8, 2], { z0.b-z3.b }, z13.b[1]\n"
        ".inst 0xc15d94a3  // sdot za.s[x8, 3], { z4.b-z7.b }, z13.b[1]\n"
        "ble 29f\n"
        ".inst 0xa0400215  // ldnt1b { z20.b-z21.b }, pn8.b/Z, [x16]\n"
        "subs x11, x11, #0x4\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa0400337  // ldnt1b { z22.b-z23.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa0400369  // ldnt1b { z8.b-z9.b }, pn8.b/Z, [x27]\n"
        "addvl x27, x27, #2\n"
        ".inst 0xa040030b  // ldnt1b { z10.b-z11.b }, pn8.b/Z, [x24]\n"
        "addvl x24, x24, #2\n"
        ".inst 0xa0400125  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x9]\n"
        ".inst 0xc15d9aa0  // sdot za.s[x8, 0], { z20.b-z23.b }, z13.b[2]\n"
        "addvl x9, x9, #2\n"
        ".inst 0xa04002c7  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x22]\n"
        "addvl x22, x22, #2\n"
        ".inst 0xa0400391  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x28]\n"
        ".inst 0xc15d9921  // sdot za.s[x8, 1], { z8.b-z11.b }, z13.b[2]\n"
        "addvl x28, x28, #2\n"
        ".inst 0xa0400353  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x26]\n"
        "addvl x26, x26, #2\n"
        ".inst 0xc15d98a2  // sdot za.s[x8, 2], { z4.b-z7.b }, z13.b[2]\n"
        ".inst 0xc15d9a23  // sdot za.s[x8, 3], { z16.b-z19.b }, z13.b[2]\n"
        "ble 29f\n"
        ".inst 0xa0400205  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x16]\n"
        "addvl x16, x16, #2\n"
        ".inst 0xa0400327  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x25]\n"
        "addvl x25, x25, #2\n"
        ".inst 0xa0400361  // ldnt1b { z0.b-z1.b }, pn8.b/Z, [x27]\n"
        ".inst 0xa0400303  // ldnt1b { z2.b-z3.b }, pn8.b/Z, [x24]\n"
        ".inst 0xa0400131  // ldnt1b { z16.b-z17.b }, pn8.b/Z, [x9]\n"
        ".inst 0xc15d9ca0  // sdot za.s[x8, 0], { z4.b-z7.b }, z13.b[3]\n"
        ".inst 0xa04002d3  // ldnt1b { z18.b-z19.b }, pn8.b/Z, [x22]\n"
        ".inst 0xa0400385  // ldnt1b { z4.b-z5.b }, pn8.b/Z, [x28]\n"
        ".inst 0xc15d9c21  // sdot za.s[x8, 1], { z0.b-z3.b }, z13.b[3]\n"
        ".inst 0xa0400347  // ldnt1b { z6.b-z7.b }, pn8.b/Z, [x26]\n"
        ".inst 0xc15d9e22  // sdot za.s[x8, 2], { z16.b-z19.b }, z13.b[3]\n"
        ".inst 0xc15d9ca3  // sdot za.s[x8, 3], { z4.b-z7.b }, z13.b[3]\n"
        "29:"  // Width 4: Multiply loop: multiply skip
        "tbnz %x[flags], #31, 30f\n"
        "sdot z25.s, z13.b, z27.b\n"
        "30:"  // Width 4: Multiply loop: unique 8: skip row sum
        ".inst 0xc0060c04  // mova { z4.d-z7.d }, za.d[x8, #0]\n"
        ".inst 0xa0404202  // ld1w { z2.s-z3.s }, pn8.b/Z, [x16]\n"
        "add x22, %x[k_args], %[c_offset]\n"
        "add x21, %x[k_args], %[minval]\n"
        ".inst 0xa040432c  // ld1w { z12.s-z13.s }, pn8.b/Z, [x25]\n"
        "add x20, %x[k_args], %[maxval]\n"
        ".inst 0xc0060c3c  // mova { z28.d-z31.d }, za.d[x8, #1]\n"
        "add x16, x16, x12, LSL #1\n"
        "ld1rw { z0.s }, p2/Z, [x22]\n"
        "add x25, x25, x12, LSL #1\n"
        ".inst 0xc0060c54  // mova { z20.d-z23.d }, za.d[x8, #2]\n"
        "ld1rw { z1.s }, p2/Z, [x21]\n"
        ".inst 0xc0060c68  // mova { z8.d-z11.d }, za.d[x8, #3]\n"
        ".inst 0xc132e084  // scvtf { z4.s-z7.s }, { z4.s-z7.s }\n"
        "ld1rw { z17.s }, p2/Z, [x20]\n"
        "fmul z4.s, z4.s, z2.s\n"
        "fmul z5.s, z5.s, z3.s\n"
        ".inst 0xc132e39c  // scvtf { z28.s-z31.s }, { z28.s-z31.s }\n"
        "fmul z6.s, z6.s, z12.s\n"
        "fmul z7.s, z7.s, z13.s\n"
        ".inst 0xc132e294  // scvtf { z20.s-z23.s }, { z20.s-z23.s }\n"
        ".inst 0xc132e108  // scvtf { z8.s-z11.s }, { z8.s-z11.s }\n"
        ".inst 0xc1b8e084  // frintn { z4.s-z7.s }, { z4.s-z7.s }\n"
        ".inst 0xc131e084  // fcvtzs { z4.s-z7.s }, { z4.s-z7.s }\n"
        ".inst 0xc1a0ab04  // add { z4.s-z7.s }, { z4.s-z7.s }, z0.s\n"
        ".inst 0xc1b1cc24  // sclamp { z4.s-z7.s }, z1.s, z17.s\n"
        "uzp1 z4.h, z4.h, z5.h\n"
        "uzp1 z16.h, z6.h, z7.h\n"
        "uzp1 z4.b, z4.b, z16.b\n"
        "st1b { z4.b }, p2, [x14]\n"
        ".inst 0xa1404212  // ld1w { z18.s, z26.s }, pn8.b/Z, [x16]\n"
        "add x16, x16, x12, LSL #1\n"
        ".inst 0xa0404324  // ld1w { z4.s-z5.s }, pn8.b/Z, [x25]\n"
        "add x25, x25, x12, LSL #1\n"
        "fmul z28.s, z28.s, z18.s\n"
        "fmul z29.s, z29.s, z26.s\n"
        "fmul z30.s, z30.s, z4.s\n"
        "fmul z31.s, z31.s, z5.s\n"
        ".inst 0xc1b8e39c  // frintn { z28.s-z31.s }, { z28.s-z31.s }\n"
        ".inst 0xc131e39c  // fcvtzs { z28.s-z31.s }, { z28.s-z31.s }\n"
        ".inst 0xc1a0ab1c  // add { z28.s-z31.s }, { z28.s-z31.s }, z0.s\n"
        ".inst 0xc1b1cc3c  // sclamp { z28.s-z31.s }, z1.s, z17.s\n"
        "uzp1 z28.h, z28.h, z29.h\n"
        "uzp1 z16.h, z30.h, z31.h\n"
        "uzp1 z28.b, z28.b, z16.b\n"
        "st1b { z28.b }, p2, [x14, #1, MUL VL]\n"
        ".inst 0xa1404207  // ld1w { z7.s, z15.s }, pn8.b/Z, [x16]\n"
        "add x16, x16, x12, LSL #1\n"
        ".inst 0xa1404324  // ld1w { z4.s, z12.s }, pn8.b/Z, [x25]\n"
        "add x25, x25, x12, LSL #1\n"
        "fmul z20.s, z20.s, z7.s\n"
        "fmul z21.s, z21.s, z15.s\n"
        "fmul z22.s, z22.s, z4.s\n"
        "fmul z23.s, z23.s, z12.s\n"
        ".inst 0xc1b8e294  // frintn { z20.s-z23.s }, { z20.s-z23.s }\n"
        ".inst 0xc131e294  // fcvtzs { z20.s-z23.s }, { z20.s-z23.s }\n"
        ".inst 0xc1a0ab14  // add { z20.s-z23.s }, { z20.s-z23.s }, z0.s\n"
        ".inst 0xc1b1cc34  // sclamp { z20.s-z23.s }, z1.s, z17.s\n"
        "uzp1 z20.h, z20.h, z21.h\n"
        "uzp1 z16.h, z22.h, z23.h\n"
        "uzp1 z20.b, z20.b, z16.b\n"
        "st1b { z20.b }, p2, [x14, #2, MUL VL]\n"
        ".inst 0xa1404206  // ld1w { z6.s, z14.s }, pn8.b/Z, [x16]\n"
        ".inst 0xa1404327  // ld1w { z7.s, z15.s }, pn8.b/Z, [x25]\n"
        "fmul z8.s, z8.s, z6.s\n"
        "fmul z9.s, z9.s, z14.s\n"
        "fmul z10.s, z10.s, z7.s\n"
        "fmul z11.s, z11.s, z15.s\n"
        ".inst 0xc1b8e108  // frintn { z8.s-z11.s }, { z8.s-z11.s }\n"
        ".inst 0xc131e108  // fcvtzs { z8.s-z11.s }, { z8.s-z11.s }\n"
        ".inst 0xc1a0ab08  // add { z8.s-z11.s }, { z8.s-z11.s }, z0.s\n"
        ".inst 0xc1b1cc28  // sclamp { z8.s-z11.s }, z1.s, z17.s\n"
        "uzp1 z8.h, z8.h, z9.h\n"
        "uzp1 z16.h, z10.h, z11.h\n"
        "uzp1 z8.b, z8.b, z16.b\n"
        "st1b { z8.b }, p1, [x14, #3, MUL VL]\n"
        "addvl x14, x14, #4\n"
        "subs x13, x13, #0x4\n"
        "mov x16, x23\n"
        "sub %x[N], %x[N], x15, LSL #2\n"
        "bgt 4b\n"
        "32:"  // Exit
        ".inst 0xd503467f  // SMSTOP\n"
        : [N] "+&r"(N), [flags] "+&r"(flags)
        : [A_ptr] "r"(A_ptr), [B_ptr] "r"(B_ptr), [K] "r"(K), [c_offset] "I"(offsetof(KernelArgs, c_offset)),
          [k_args] "r"(&k_args), [maxval] "I"(offsetof(KernelArgs, maxval)), [minval] "I"(offsetof(KernelArgs, minval)),
          [output_ptr] "r"(output_ptr)
        : "cc", "memory", "p0", "p1", "p10", "p11", "p12", "p13", "p14", "p15", "p2", "p3", "p4", "p5", "p6", "p7",
          "p8", "p9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x20", "x21", "x22", "x23", "x24", "x25", "x26",
          "x27", "x28", "x8", "x9", "z0", "z1", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19",
          "z2", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z3", "z30", "z31", "z4", "z5",
          "z6", "z7", "z8", "z9");
}

#endif  // Architectural features check.
